package com.tim.test.download;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * jQuery选择器规则，https://www.runoob.com/jquery/jquery-ref-selectors.html：
 * 	.选择class，#选择id，[]选择属性，:contains(第)内容包含‘第’
 * 	p,h2；同时选择p、h2
 * 	div.main；同时满足
 * 	div .comment；子孙
 * 	div>p；子级
 * 	div~h1；兄弟
 */
public class NovelHtmlAnt2 {
	// 解析地址
	String firstPage = "https://www.xsla.cc/45066/1.html";
	int executeJs = 0;//0不执行js，1执行js
	// 链接url前缀
	String urlPre = "https://www.xsla.cc";//默认host
	String titleCss = "";
	String contentContainerCss = "";//默认body
	String contentElementCss = "#content";
	String nextPageCss = "div.bottem2 a";// 下一页
	String nextPageCss2 = "";// 下一章；处理下一页、下一章不同

	/**不变*/
	// 目标文件夹
	static String dir = "D:/个人资料/书籍/novel/";
	String pageFileName = "pageFileName"+ System.currentTimeMillis()+".txt";
	String chapterTitle ="";
	int skipMultiTag = 0;
	Charset charset = StandardCharsets.UTF_8;
	WebClient webClient;

	@Test
	public void downloadByPage() throws Exception {
		// 初始化参数、文件
		if(StringUtils.isBlank(urlPre)){
			urlPre = firstPage.substring(0, firstPage.lastIndexOf("/", firstPage.indexOf("//")+2));
		}
		if(StringUtils.isBlank(contentContainerCss)){
			contentContainerCss = "body";
		}
		File dirFile = new File(dir);
		if(!dirFile.exists()){
			dirFile.mkdirs();
		}
		File file = new File(dir+"/"+pageFileName);
		if(!file.exists()){
			file.createNewFile();
		}

		downloadByPage(firstPage, urlPre);

		System.out.println("SUCCESS");
		if (webClient!=null){
			webClient.close();
		}
	}

	private Object downloadByPage(String href, String urlPre) throws Exception{
		System.out.println(href);
		Document document;
		if(executeJs==1){
			String html = executeJs(href);
			document = Jsoup.parse(new ByteArrayInputStream(html.getBytes(charset)), charset.name(), "");
		}else {
			document = Jsoup.connect(href).get();
		}

		/* step，获取标题内容*/
		StringBuilder builder = new StringBuilder();
		if(StringUtils.isNotBlank(titleCss)){
			Elements titleElements = document.body().select(titleCss);
			if(titleElements!=null&& !titleElements.isEmpty() && !chapterTitle.equals(titleElements.first().text())){
				chapterTitle = titleElements.first().text();
				builder.append(chapterTitle).append("\n");
			}
		}

		/* step，获取章节内容*/
		// 初筛
		Elements containers = document.body().select(contentContainerCss);
		// 选中目标元素
		Elements targetTags = containers.select(contentElementCss);
		// 获取内容
		for (Element targetTag : targetTags){
			if(targetTag.childrenSize()>0 && skipMultiTag==1){
				continue;
			}
			String text = getText(targetTag).trim();
			builder.append(text).append("\n");
		}
		String content = builder.toString()
				.replaceAll("\n+", "\n");

		/*step，追加写入文档*/
		Files.write(Paths.get(dir,pageFileName),
				content.getBytes(charset),
				StandardOpenOption.APPEND
		);

		/* step，获取下一页链接；无则终止*/
		Element nextPage = getNextPage(document, nextPageCss);
		String nextPageHref =null;
		if(nextPage!=null){
			nextPageHref = nextPage.attr("href");
		}else if(StringUtils.isNotBlank(nextPageCss2)){
			nextPage = getNextPage(document, nextPageCss2);
			if(nextPage!=null){
				nextPageHref = nextPage.attr("href");
			}
		}
		if(StringUtils.isNotBlank(nextPageHref)){
			nextPageHref = nextPageHref.startsWith("http") ? nextPageHref : urlPre + nextPageHref;
			return downloadByPage(nextPageHref, urlPre);
		}
		return null;
	}

	private String executeJs(String url){
		if(webClient==null){
			webClient = new WebClient(BrowserVersion.CHROME);//新建一个模拟谷歌Chrome浏览器的浏览器客户端对象
			webClient.getOptions().setThrowExceptionOnScriptError(false);//当JS执行出错的时候是否抛出异常, 这里选择不需要
			webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);//当HTTP的状态非200时是否抛出异常, 这里选择不需要
			webClient.getOptions().setActiveXNative(false);//不启用ActiveX
			webClient.getOptions().setCssEnabled(false);//是否启用CSS, 因为不需要展现页面, 所以不需要启用
			webClient.getOptions().setJavaScriptEnabled(true); //很重要，启用JS
			webClient.getOptions().setDownloadImages(false);//不下载图片
			webClient.setAjaxController(new NicelyResynchronizingAjaxController());//很重要，设置支持AJAX
		}
		HtmlPage page = null;
		try {
			page = webClient.getPage(url);//尝试加载上面图片例子给出的网页
		} catch (Exception e) {
			e.printStackTrace();
		}

		webClient.waitForBackgroundJavaScript(20*1000);//异步JS执行需要耗时,所以这里线程要阻塞30秒,等待异步JS执行结束

		charset = page.getCharset();
		String pageXml = page.asXml();//直接将加载完成的页面转换成xml格式的字符串
//		Document parse = Jsoup.parse(pageXml);
//		System.out.println(parse);
		return pageXml;
	}

	private String getText(Element child) {
		String s = child.html()
				.replaceAll("(<br>)+", "\n")
				.replaceAll("(<script>[\\w\\W]*</script>)", "")
				.replaceAll("\n+", "\n")
				.replaceAll("&nbsp;", " ");
		return s;

	}

	private Element getNextPage(Document document, String nextPageCss) {
		Elements nextPages = document.body().select(nextPageCss);
		Element nextPage = nextPages.stream()
				.filter(e -> {
					String text = e.text();
					return StringUtils.isNotBlank(text) && text.trim().contains("下一");
				}).findFirst().orElse(null);
		return nextPage;
	}

	public static void main(String[] args) throws IOException {
		String filePath = "D:/aa/pageFileName1746959087734.txt";
		String[] filters = new String[]{
				"正在手打中",
				"近期更新缓慢"
		};

		//读取内容，过滤写回
		Path path = Paths.get(filePath);
		List<String> list;
		try{
			list = Files.readAllLines(path);
		}catch (MalformedInputException e){
			list = Files.readAllLines(path, Charset.forName("gb2312"));
		}
		String content = list.stream()
				.filter(e -> !StringUtils.startsWithAny(e.trim(),filters))
				.map(String::trim)
				.collect(Collectors.joining("\n"));
		content = content.replaceAll("((<p>)|(</p>)|(<br>)|(<br/>))", "")
				.replaceAll("((<b>)|(</b>))", "")
				.replaceAll("　　", "")
				.replaceAll("・", ".")
				.replaceAll(" {2,}", "  ")
				.replaceAll("(第[一二三四五六七八九十百零]+章)","\n$1\n")
				.replaceAll("guī_tóu", "龟头")
				.replaceAll("yīn_jīng", "阴茎")
				.replaceAll("rǔ_fáng", "乳房")
				.replaceAll("háo_rǔ", "豪乳")
				.replaceAll("jī_bā", "鸡巴")
				.replaceAll("ròu_bàng", "肉棒")
				.replaceAll("ròu_dòng", "肉洞")
				.replaceAll("yīn_chún", "阴唇")
				.replaceAll("chōu_chā", "抽查")
				.replaceAll("jīng_yè", "精液")
				.replaceAll("shè_jīng", "射精")
				.replaceAll("jù_rǔ", "巨乳")
				.replaceAll("yín_shuǐ", "淫水")
				.replaceAll("sāo_xué", "骚穴")
				.replaceAll("shú_nǚ", "熟女")
				.replaceAll("yù_wàng", "欲望")
				.replaceAll("yīn_máo", "阴毛")
				.replaceAll("ròu_tǐ", "肉体")
				.replaceAll("yín_xué", "淫穴")
				.replaceAll("yín_dàng", "淫荡")
				.replaceAll("mì_xué", "蜜穴")
				.replaceAll("xìng_jiāo", "性交")
				.replaceAll("pēn_shè", "喷射")
				.replaceAll("luǒ_tǐ", "裸体")
				.replaceAll("sān_jí", "三级")
				.replaceAll("yīn_dào", "阴道")
						.replaceAll("cháo_chuī", "潮水")
				.replaceAll("chù_nǚ", "处女")
				.replaceAll("chā_wǒ", "插我")
				.replaceAll("cuī_qíng_yào，", "催情药")
				.replaceAll("cǎi_huā", "采花")
				.replaceAll("dà_rǔ", "大乳")
				.replaceAll("dàng_fù", "荡妇")
				.replaceAll("dǎ_shǒu_qiāng", "打手枪")
				.replaceAll("fēng_liú", "风流")
				.replaceAll("gāng_jiāo", "肛交")
				.replaceAll("gāng_mén", "肛门")
				.replaceAll("gāo_wán", "睾丸")
				.replaceAll("jìn_guǒ", "禁果")
				.replaceAll("jī_ba", "鸡巴")
				.replaceAll("jī_bachā_nǐ", "鸡巴插你")
				.replaceAll("jī_jī", "鸡鸡")
				.replaceAll("kǒu_jiāo", "口交")
				.replaceAll("luàn_lún", "乱伦")
				.replaceAll("làn_jiāo", "滥交")
				.replaceAll("làng_jiào", "浪叫")
				.replaceAll("mí_hún_yào", "迷魂药")
				.replaceAll("mí_jiān", "迷奸")
				.replaceAll("pò_chù", "破处")
				.replaceAll("qiáng_jiān", "强奸")
				.replaceAll("qín_shòu", "禽兽")
				.replaceAll("qíng_sè", "情色")
				.replaceAll("ròu_gùn", "肉棍")
				.replaceAll("ròu_yù", "肉欲")
				.replaceAll("rǔ_qiú", "乳球")
				.replaceAll("rǔ_tóu", "乳头")
				.replaceAll("shuāng_rǔ", "双乳")
				.replaceAll("shòu_yù", "兽欲")
				.replaceAll("shǒu_yín", "手淫")
				.replaceAll("sè_láng", "色狼")
				.replaceAll("sè_mó", "色魔")
				.replaceAll("sè_qíng", "色情")
				.replaceAll("sè_qíng_wǎng_zhàn，", "色情网站")
				.replaceAll("sāo_huò", "骚货")
				.replaceAll("sū_xiōng", "酥胸")
				.replaceAll("tuō_guāng", "脱光")
				.replaceAll("tōu_huān", "偷欢")
				.replaceAll("xiǎo_xué", "小穴")
				.replaceAll("xìng_gāo_cháo", "性高潮")
				.replaceAll("xìng_sāo_rǎo", "性骚扰")
				.replaceAll("xìng_ài", "性爱")
				.replaceAll("yín_luàn", "淫乱")
				.replaceAll("yín_mó", "淫魔")
				.replaceAll("yín_tài", "淫态")
				.replaceAll("yín_wádàng_fù", "淫娃荡妇")
				.replaceAll("yù_nǚ", "玉女")
				.replaceAll("yù_rǔ", "玉乳")
				.replaceAll("yù_tuǐ", "玉腿")
				.replaceAll("yīn_bù", "阴部")
				.replaceAll("yīn_dì", "阴蒂")
				.replaceAll("yīn_fù", "阴阜")
				.replaceAll("yīn_hù", "阴户")
				.replaceAll("yīn_mén，", "阴门")
				.replaceAll("yīn_náng", "阴囊")
		;
		Files.write(path,
				content.getBytes());
	}

	@Test
	public void find() throws IOException {
		String regExp = "([A-z][^\\w]?)+";

		String filePath = "D:/aa/pageFileName1746959087734.txt";
		String[] filters = new String[]{
				"正在手打中",
				"近期更新缓慢"
		};

		//读取内容，过滤写回
		Path path = Paths.get(filePath);
		List<String> list;
		try{
			list = Files.readAllLines(path);
		}catch (MalformedInputException e){
			list = Files.readAllLines(path, Charset.forName("gb2312"));
		}
		String content = list.stream()
				.filter(e -> !StringUtils.startsWithAny(e.trim(),filters))
				.map(String::trim)
				.collect(Collectors.joining("\n"));

		Matcher matcher = Pattern.compile(regExp).matcher(content);
		Set<String> set=new TreeSet<>();
		while (matcher.find()){
				String group = matcher.group();
			if(group.length()>3) {
				set.add(group.replaceAll("[\\u4e00-\\u9fa5]", ""));
			}
		}
		set.stream()
				.map(e->String.format(".replaceAll(\"%s\", \"\")",e))
			.forEach(System.out::println);
	}


}
